#!/usr/bin/perl -ws

use Time::Local;
use POSIX qw(ceil floor);
use Tie::Hash;
#use GDBM_File;
use SDBM_File;
use Fcntl;

$WORKINPROGRESS = "workinprogress";
$WORKINPROGRESS_LEN = length(WORKINPROGRESS);

require "$ENV{'HOME'}/bin/fileutl.pm";

if (defined($t) && ($t ne "")) {
	$trace = $t;
} else {
	$trace = 0;
}
if (!defined($short_test) || ($short_test eq "")) {
	$short_test = 0;
}
if (!defined($max_short_test) || ($max_short_test eq "")) {
	$max_short_test = 0;
}
if (!defined($testcount) || ($testcount eq "")) {
	$testcount = 0;
}
if (!defined($file_pattern) || ($file_pattern eq "")) {
	print "Error.  File pattern '-file_pattern=' not specified.\n";
	exit 1;
}

if (!defined($hourly_report) || ($hourly_report eq "")) {
	$hourly_report = 0
}

if (!defined($diagnostic_file) || ($diagnostic_file eq "")) {
	print "Error.  Dianostic file '-diagnostic_file=' not specified.\n";
	exit 1;
}

if (defined($bulkins_list) && ($bulkins_list eq "")) {
	print "bulkins_list is empty, ignoring.\n";
	undef $bulkins_list;
}

if (!defined($sdbm_file) || ($sdbm_file eq "")) {
	print "Error.  Temp database file(Needs to be on mount point with " .
		  "a lot of free space.  Do not specify on /tmp). '-sdbm_file=' " .
		  "not specified.\n";
	exit 1;
}

#TODO
#if (! -w $sdbm_file ) {
#	print "Error.  Temp database file needs to be write accessible.\n";
#	exit 1;
#}

&open_it( \*DIAGFILE, ">", $diagnostic_file);

if (defined($bulkins_list)) {
	&open_it( \*BULKLISTFILE, "<", $bulkins_list);
}

%key_names = ();
%key_descriptions = ();
%value_names = ();
%value_descriptions = ();
%assoc_symbols = ();
%open_bif_names = ();
$assoc_symbols{\%open_bif_names} = \open_bif_names;
$key_names{\%open_bif_names} = "coll";
$key_descriptions{\%open_bif_names} = "Collection";
$value_names{\%open_bif_names} = "file";
$value_descriptions{\%open_bif_names} = "Bif file name";

%bif_name_counts = ();
$assoc_symbols{\%bif_name_counts} = \bif_name_counts;
$key_names{\%bif_name_counts} = "file";
$key_descriptions{\%bif_name_counts} = "Bif file name";
$value_names{\%bif_name_counts} = "count";
$value_descriptions{\%bif_name_counts} = "Bif file doc counts";

%act_bif_name_counts = ();
$assoc_symbols{\%act_bif_name_counts} = \act_bif_name_counts;
$key_names{\%act_bif_name_counts} = "file";
$key_descriptions{\%act_bif_name_counts} = "Bif file name";
$value_names{\%act_bif_name_counts} = "count";
$value_descriptions{\%act_bif_name_counts} = "Act bif file doc counts";

%bif_name_colls = ();
$assoc_symbols{\%bif_name_colls} = \bif_name_colls;
$key_names{\%bif_name_colls} = "file";
$key_descriptions{\%bif_name_colls} = "Bif file name";
$value_names{\%bif_name_colls} = "coll";
$value_descriptions{\%bif_name_colls} = "Bif file collection";

%bif_name_proc = ();
$assoc_symbols{\%bif_name_proc} = \bif_name_proc;
$key_names{\%bif_name_proc} = "file";
$key_descriptions{\%bif_name_proc} = "Bif file name";
$value_names{\%bif_name_proc} = "processed";
$value_descriptions{\%bif_name_proc} = "Bif file processed (indexed = 1).";

%bif_name_date_created = ();
$assoc_symbols{\%bif_name_date_created} = \bif_name_date_created;
$key_names{\%bif_name_date_created} = "file";
$key_descriptions{\%bif_name_date_created} = "Bif file name";
$value_names{\%bif_name_date_created} = "date";
$value_descriptions{\%bif_name_date_created} = "Date bif file created.";

%act_bif_name_date_created = ();
$assoc_symbols{\%act_bif_name_date_created} = \act_bif_name_date_created;
$key_names{\%act_bif_name_date_created} = "file";
$key_descriptions{\%act_bif_name_date_created} = "Bif file name";
$value_names{\%act_bif_name_date_created} = "date";
$value_descriptions{\%act_bif_name_date_created} = "Actual date bif file created.";

%bif_name_hour_created = ();
$assoc_symbols{\%bif_name_hour_created} = \bif_name_hour_created;
$key_names{\%bif_name_hour_created} = "file";
$key_descriptions{\%bif_name_hour_created} = "Bif file name";
$value_names{\%bif_name_hour_created} = "hour";
$value_descriptions{\%bif_name_hour_created} = "Hour bif file created.";

%bif_name_date_closed = ();
$assoc_symbols{\%bif_name_date_closed} = \bif_name_date_closed;
$key_names{\%bif_name_date_closed} = "file";
$key_descriptions{\%bif_name_date_closed} = "Bif file name";
$value_names{\%bif_name_date_closed} = "date";
$value_descriptions{\%bif_name_date_closed} = "Date bif file closed.";

%indexer_fatal_shutdown = ();
$assoc_symbols{\%indexer_fatal_shutdown} = \indexer_fatal_shutdown;
$key_names{\%indexer_fatal_shutdown} = "date_time_min_sec";
$key_descriptions{\%indexer_fatal_shutdown} = "Date/time/minute/sec of fatal exit";
$value_names{\%indexer_fatal_shutdown} = "exception";
$value_descriptions{\%indexer_fatal_shutdown} = "Exception of fatal exit.";

%bif_name_hour_closed = ();
$assoc_symbols{\%bif_name_hour_closed} = \bif_name_hour_closed;
$key_names{\%bif_name_hour_closed} = "file";
$key_descriptions{\%bif_name_hour_closed} = "Bif file name";
$value_names{\%bif_name_hour_closed} = "hour";
$value_descriptions{\%bif_name_hour_closed} = "Hour bif file closed.";

%bif_name_date_indexed = ();
$assoc_symbols{\%bif_name_date_indexed} = \bif_name_date_indexed;
$key_names{\%bif_name_date_indexed} = "file";
$key_descriptions{\%bif_name_date_indexed} = "Bif file name";
$value_names{\%bif_name_date_indexed} = "date";
$value_descriptions{\%bif_name_date_indexed} = "Date bif file indexed.";

%bif_name_date_started_indexing = ();
$assoc_symbols{\%bif_name_date_started_indexing} = \bif_name_date_started_indexing;
$key_names{\%bif_name_date_started_indexing} = "file";
$key_descriptions{\%bif_name_date_started_indexing} = "Bif file name";
$value_names{\%bif_name_date_started_indexing} = "date";
$value_descriptions{\%bif_name_date_started_indexing} = "Date bif file indexed.";

%bif_name_hour_indexed = ();
$assoc_symbols{\%bif_name_hour_indexed} = \bif_name_hour_indexed;
$key_names{\%bif_name_hour_indexed} = "file";
$key_descriptions{\%bif_name_hour_indexed} = "Bif file name";
$value_names{\%bif_name_hour_indexed} = "hour";
$value_descriptions{\%bif_name_hour_indexed} = "Hour bif file indexed.";

%bif_name_hour_started_indexing = ();
$assoc_symbols{\%bif_name_hour_started_indexing} = \bif_name_hour_started_indexing;
$key_names{\%bif_name_hour_started_indexing} = "file";
$key_descriptions{\%bif_name_hour_started_indexing} = "Bif file name";
$value_names{\%bif_name_hour_started_indexing} = "hour";
$value_descriptions{\%bif_name_hour_started_indexing} = "Hour bif file indexed.";

%curr_doc_ix = ();
$assoc_symbols{\%curr_doc_ix} = \curr_doc_ix;
$key_names{\%curr_doc_ix} = "thread";
$key_descriptions{\%curr_doc_ix} = "Thread of collection index.";
$value_names{\%curr_doc_ix} = "collix";
$value_descriptions{\%curr_doc_ix} = "Collection index.";

%gather_counts = ();
$assoc_symbols{\%gather_counts} = \gather_counts;
$key_names{\%gather_counts} = "coll:date:vlang";
$key_descriptions{\%gather_counts} =
	"Collection gathered docs gathered on date." .
	"Date gathered docs.:" .
	"Language docs gathered on date";
$value_names{\%gather_counts} = "count";
$value_descriptions{\%gather_counts} = "Gather doc counts";

%gather_bif_date_counts = ();
$assoc_symbols{\%gather_bif_date_counts} = \gather_bif_date_counts;
$key_names{\%gather_bif_date_counts} = "coll:date:vlang";
$key_descriptions{\%gather_bif_date_counts} =
	"Collection gathered bif files gathered on date." .
	"Date gathered bif files.:" .
	"Language bif files gathered on date";
$value_names{\%gather_bif_date_counts} = "count";
$value_descriptions{\%gather_bif_date_counts} = "Gather doc counts";

%gather_bif_hour_counts = ();
$assoc_symbols{\%gather_bif_hour_counts} = \gather_bif_hour_counts;
$key_names{\%gather_bif_hour_counts} = "coll:date:vlang";
$key_descriptions{\%gather_bif_hour_counts} =
	"Collection gathered bif files gathered on date." .
	"Date gathered bif files.:" .
	"Language bif files gathered on date";
$value_names{\%gather_bif_hour_counts} = "count";
$value_descriptions{\%gather_bif_hour_counts} = "Gather doc counts";

%coll_threads = ();
$assoc_symbols{\%coll_threads} = \coll_threads;
$key_names{\%coll_threads} = "coll";
$key_descriptions{\%coll_threads} = "Collection of thread.";
$value_names{\%coll_threads} = "thread";
$value_descriptions{\%coll_threads} = "Thread the collection was running on.";

%prev_coll_threads = ();
$assoc_symbols{\%prev_coll_threads} = \prev_coll_threads;
$key_names{\%prev_coll_threads} = "coll";
$key_descriptions{\%prev_coll_threads} = "Collection of thread.";
$value_names{\%prev_coll_threads} = "thread";
$value_descriptions{\%prev_coll_threads} = "Thread the collection was running on.";

%indexed_date_counts = ();
$assoc_symbols{\%indexed_date_counts} = \indexed_date_counts;
$key_names{\%indexed_date_counts} =  "coll:date:vlang:thread";
$key_descriptions{\%indexed_date_counts} =
	"Collection indexed docs on date." .
	"Date indexed docs.:" .
	"Language docs indexed on date" .
	"Thread docs indexed on date";
$value_names{\%indexed_date_counts} = "count";
$value_descriptions{\%indexed_date_counts} = "Indexed doc counts";

%indexed_hour_counts = ();
#tie(%indexed_hour_counts, 'GDBM_File', $sdbm_file, &GDBM_WRCREAT, 0664);
tie(%indexed_hour_counts, 'SDBM_File', $sdbm_file, 1, 0);
$assoc_symbols{\%indexed_hour_counts} = \indexed_hour_counts;
$key_names{\%indexed_hour_counts} =  "coll:hour:vlang:thread";
$key_descriptions{\%indexed_hour_counts} =
	"Collection indexed docs gathered on hour." .
	"Hour indexed docs.:" .
	"Language docs indexed on hour" .
	"Thread docs indexed on hour";
$value_names{\%indexed_hour_counts} = "count";
$value_descriptions{\%indexed_hour_counts} = "Indexed doc counts";

%indexed_bif_date_counts = ();
$assoc_symbols{\%indexed_bif_date_counts} = \indexed_bif_date_counts;
$key_names{\%indexed_bif_date_counts} = "date:vlang:coll:thread";
$key_descriptions{\%indexed_bif_date_counts} =
	"Collection indexed bif files gathered on date." .
	"Date indexed bif files.:" .
	"Language bif files indexed on date" .
	"Thread bif files indexed on date";
$value_names{\%indexed_bif_date_counts} = "count";
$value_descriptions{\%indexed_bif_date_counts} = "Indexed bif file counts";

%indexed_bif_hour_counts = ();
$assoc_symbols{\%indexed_bif_hour_counts} = \indexed_bif_hour_counts;
$key_names{\%indexed_bif_hour_counts} = "hour:vlang:coll:thread";
$key_descriptions{\%indexed_bif_hour_counts} =
	"Collection indexed bif files gathered on hour." .
	"Hour indexed bif files.:" .
	"Language bif files indexed on hour" .
	"Thread bif files indexed on hour";
$value_names{\%indexed_bif_hour_counts} = "count";
$value_descriptions{\%indexed_bif_hour_counts} = "Indexed bif file counts";

%lost_date_counts = ();
$assoc_symbols{\%lost_date_counts} = \lost_date_counts;
$key_names{\%lost_date_counts} =  "coll:date:vlang:thread";
$key_descriptions{\%lost_date_counts} =
	"Collection lost docs gathered on date." .
	"Date lost docs.:" .
	"Language docs lost on date" .
	"Thread docs lost on date";
$value_names{\%lost_date_counts} = "count";
$value_descriptions{\%lost_date_counts} = "Lost doc counts";

%lost_bif_date_counts = ();
$assoc_symbols{\%lost_bif_date_counts} = \lost_bif_date_counts;
$key_names{\%lost_bif_date_counts} =  "coll:date:vlang:thread";
$key_descriptions{\%lost_bif_date_counts} =
	"Collection lost docs gathered on date." .
	"Date lost docs.:" .
	"Language docs lost on date" .
	"Thread docs lost on date";
$value_names{\%lost_bif_date_counts} = "count";
$value_descriptions{\%lost_bif_date_counts} = "Lost bif counts";

%vlang_counts = ();
$assoc_symbols{\%vlang_counts} = \vlang_counts;
$key_names{\%vlang_counts} = "Language";
$key_descriptions{\%vlang_counts} = "Language of the collection counts";
$value_names{\%vlang_counts} = "count";
$value_descriptions{\%vlang_counts} = "Language collection counts";
$vlang_counts{'en'} = 8;
$vlang_counts{'it'} = 2;
$vlang_counts{'fr'} = 2;
$vlang_counts{'de'} = 2;
$vlang_counts{'es'} = 2;
$vlang_counts{'ja'} = 2;
$vlang_counts{'ko'} = 2;
$vlang_counts{'zh'} = 2;
$vlang_counts{'zt'} = 2;
$vlang_counts{'pt'} = 2;

&test_init();

#$mtrace = 5; #trace undo
#$trace = $mtrace; #trace undo
#$short_test = 0; #trace undo
#$max_short_test = 20; #trace undo
@files = sort(`ls -1 $file_pattern `);
print "qualifing files=@files\n";

if (defined($bulkins_list)) {
	$trace = 3;
	&read_bulkins_list();
	exit 0; #undo
};

#$testcount = 1; #trace undo
#$use_testdata = 0; #trace undo
if (defined($use_testdata)) {
	@files = ("/home/irvb/seeker/logs/ungziped/standardvicincr.log.2007-07-23");
	foreach $log_file (@files) {
		$line_no = 0;
		$shutdown = 0;
		foreach $line (split("\n", $test7)) {
			$line_no++;
			&proc_log_line($line, \$continue);
		}
	}
	&print_results;
	exit 0;
}
$num_files = @files;
print "Number of files to process $num_files\n";
if (@files == 0) {
	print "Error.  0 files match pattern '$file_pattern'\n";
	exit 0;
}
foreach $log_file (@files) {
	if ($log_file =~ /standardvicincr.log.*gz$/) {
		&open_it( \*LOGFILE, "", "gunzip -c $log_file |");
	} elsif ($log_file =~ /standardvicincr.log/) {
		&open_it( \*LOGFILE, "<", $log_file);
	} else {
		&print_error("Error.  File $log_file is not incremental index file.\n");
		next;
	}

	$continue = 1;
	$line_no = 0;
	$shutdown = 0;
    while (defined ($line = <LOGFILE>) && $continue) {
		$line_no++;
		&proc_log_line($line, \$continue);
    }
	if ($testcount > 1) {
		$$ptr_continue = 0;
		last;
	 }
	if ($short_test) {
		$$ptr_continue = 0;
		last;
	}
}

&print_results;
exit 0;

sub test_init {
{
$test1 = <<'EOD'
2007-07-22 01:13:08,199 DEBUG IncrGatherFiles1 - Opening ins file_path=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
2007-07-22 01:13:08,198 DEBUG IncrGatherFiles1 - coll index=3
2007-07-22 01:13:08,394 DEBUG IncrGatherFiles1 - update key,ptr=ucr_na-QTPDBMS_3053-1-en,49
2007-07-22 02:00:00,635 DEBUG Timer Daemon - Finishing lang=en 8 files 
2007-07-22 02:00:00,849 DEBUG IncrIndexer3 - coll bif=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
2007-07-22 02:47:38,760 DEBUG IncrIndexer3 - removed obj_ix=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins|/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002b-C-del_en_3.ins
EOD
};
{
$test2 = <<'EOD'
2007-07-22 01:13:08,198 DEBUG IncrGatherFiles1 - coll index=3
2007-07-22 01:13:08,394 DEBUG IncrGatherFiles1 - update key,ptr=ucr_na-QTPDBMS_3053-1-en,49
2007-07-22 02:00:00,635 DEBUG Timer Daemon - Finishing lang=en 8 files 
2007-07-22 02:00:00,849 DEBUG IncrIndexer3 - coll bif=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
2007-07-22 02:47:38,760 DEBUG IncrIndexer3 - removed obj_ix=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
EOD
};
{
$test3 = <<'EOD'
2007-07-22 20:13:12,677 DEBUG IncrGatherFiles0 - coll index=0
2007-07-22 20:13:12,677 DEBUG IncrGatherFiles0 - Opening ins file_path=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185149188656.ins/062/1185152400121_0016a-C-upd_ja_0.ins
2007-07-22 20:13:12,731 DEBUG IncrGatherFiles0 - update key,ptr=emr_na-c00860334-2-ja,49
2007-07-22 21:00:00,718 DEBUG Timer Daemon - Finishing lang=ja 14 files 
2007-07-23 14:12:57,803 DEBUG IncrIndexer0 - coll bif=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185149188656.ins/062/1185152400121_0016a-C-upd_ja_0.ins
2007-07-23 14:19:17,777 DEBUG IncrIndexer0 - removed obj_ix=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185149188656.ins/062/1185152400121_0016a-C-upd_ja_0.ins|/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185149188656.ins/062/1185152400121_0016b-C-del_ja_0.ins
EOD
};
{
$test4 = <<'EOD'
2007-07-22 01:13:08,199 DEBUG IncrGatherFiles1 - Opening ins file_path=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
2007-07-22 01:13:08,198 DEBUG IncrGatherFiles1 - coll index=3
2007-07-22 01:13:08,394 DEBUG IncrGatherFiles1 - update key,ptr=ucr_na-QTPDBMS_3053-1-en,49
2007-07-22 02:00:00,635 DEBUG Timer Daemon - Finishing lang=en 8 files 
2007-07-22 02:00:00,849 DEBUG IncrIndexer3 - coll bif=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
2007-07-22 02:47:38,760 DEBUG IncrIndexer3 - removed obj_ix=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins|/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002b-C-del_en_3.ins
2007-07-22 20:13:12,677 DEBUG IncrGatherFiles0 - coll index=0
2007-07-22 20:13:12,677 DEBUG IncrGatherFiles0 - Opening ins file_path=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185149188656.ins/062/1185152400121_0016a-C-upd_ja_0.ins
2007-07-22 20:13:12,731 DEBUG IncrGatherFiles0 - update key,ptr=emr_na-c00860334-2-ja,49
EOD
};

{
$test5 = <<'EOD'
2007-07-22 01:13:08,199 DEBUG IncrGatherFiles1 - Opening ins file_path=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
2007-07-22 01:13:08,198 DEBUG IncrGatherFiles1 - coll index=3
2007-07-22 01:13:08,394 DEBUG IncrGatherFiles1 - update key,ptr=ucr_na-QTPDBMS_3053-1-en,49
2007-07-22 02:00:00,635 DEBUG Timer Daemon - Finishing lang=en 8 files 
2007-07-22 02:00:00,849 DEBUG IncrIndexer3 - coll bif=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
2007-07-22 02:47:38,760 DEBUG IncrIndexer3 - removed obj_ix=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins|/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002b-C-del_en_3.ins
2007-07-22 01:13:08,199 DEBUG IncrGatherFiles1 - Opening ins file_path=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0003a-C-upd_en_3.ins
2007-07-22 01:13:08,198 DEBUG IncrGatherFiles1 - coll index=3
2007-07-22 01:13:08,394 DEBUG IncrGatherFiles1 - update key,ptr=ucr_na-QTPDBMS_3054-1-en,49
2007-07-22 02:00:00,635 DEBUG Timer Daemon - Finishing lang=en 8 files 
EOD
};

{
$test6 = <<'EOD'
2007-07-22 01:13:08,199 DEBUG IncrGatherFiles1 - Opening ins file_path=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
2007-07-22 01:13:08,198 DEBUG IncrGatherFiles1 - coll index=3
2007-07-22 01:13:08,394 DEBUG IncrGatherFiles1 - update key,ptr=ucr_na-QTPDBMS_3053-1-en,49
2007-07-22 02:00:00,635 DEBUG Timer Daemon - Finishing lang=en 8 files 
2007-07-22 02:00:00,849 DEBUG IncrIndexer3 - coll bif=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
2007-07-22 02:47:38,760 DEBUG IncrIndexer3 - removed obj_ix=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins|/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002b-C-del_en_3.ins
2007-07-22 01:13:08,199 DEBUG IncrGatherFiles1 - Opening ins file_path=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0003a-C-upd_en_3.ins
2007-07-22 01:13:08,198 DEBUG IncrGatherFiles1 - coll index=3
2007-07-22 01:13:08,394 DEBUG IncrGatherFiles1 - update key,ptr=ucr_na-QTPDBMS_3054-1-en,49
2007-07-22 02:00:00,635 DEBUG Indexer Daemon - file=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0003a-C-upd_en_3.ins
EOD
};

{
$test7 = <<'EOD'
2007-07-22 01:13:08,199 DEBUG IncrGatherFiles1 - Opening ins file_path=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
2007-07-22 01:13:08,198 DEBUG IncrGatherFiles1 - coll index=3
2007-07-22 01:13:08,394 DEBUG IncrGatherFiles1 - update key,ptr=ucr_na-QTPDBMS_3053-1-en,49
2007-07-22 02:00:00,635 DEBUG Timer Daemon - Finishing lang=en 8 files 
2007-07-22 02:00:00,849 DEBUG IncrIndexer3 - coll bif=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
2007-07-22 02:47:38,760 DEBUG IncrIndexer3 - removed obj_ix=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins|/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002b-C-del_en_3.ins
2007-07-22 01:13:08,199 DEBUG IncrGatherFiles1 - Opening ins file_path=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0003a-C-upd_en_3.ins
2007-07-22 01:13:08,198 DEBUG IncrGatherFiles1 - coll index=3
2007-07-22 01:13:08,394 DEBUG IncrGatherFiles1 - update key,ptr=ucr_na-QTPDBMS_3054-1-en,49
2007-07-29 11:29:08,570 FATAL IncrIndexer1 - Unable to index with indexer
2007-07-29 11:30:08,570 FATAL IncrIndexer2 - Unable to index with indexer
2007-07-29 11:40:08,570 INFO main - verityShortHostName=ccesaw01
EOD
};

#Test remved without being indexed.
{
$test8 = <<'EOD'
2007-07-22 01:13:08,199 DEBUG IncrGatherFiles1 - Opening ins file_path=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
2007-07-22 01:13:08,198 DEBUG IncrGatherFiles1 - coll index=3
2007-07-22 01:13:08,394 DEBUG IncrGatherFiles1 - update key,ptr=ucr_na-QTPDBMS_3053-1-en,49
2007-07-22 02:00:00,635 DEBUG Timer Daemon - Finishing lang=en 8 files 
2007-07-22 02:00:00,849 DEBUG IncrIndexer3 - coll bif=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins
2007-07-22 02:47:38,760 DEBUG IncrIndexer3 - removed obj_ix=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002a-C-upd_en_3.ins|/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0002b-C-del_en_3.ins
2007-07-22 01:13:08,199 DEBUG IncrGatherFiles1 - Opening ins file_path=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0003a-C-upd_en_3.ins
2007-07-22 01:13:08,198 DEBUG IncrGatherFiles1 - coll index=3
2007-07-22 01:13:08,394 DEBUG IncrGatherFiles1 - update key,ptr=ucr_na-QTPDBMS_3054-1-en,49
2007-07-22 02:00:00,635 DEBUG Timer Daemon - Finishing lang=en 8 files 
2007-07-23 08:28:57,961 DEBUG IncrRenamer - delete file path=/var/opt/apps/verityindexer/verityindexer-2.0/data/odsbulkins/incremental/ccesaw01/topic/workinprogress/1185082272329.ins/061/1185082272329_0003a-C-upd_en_3.ins
EOD
};
}

sub read_bulkins_list {
	$trace = 4;
	my($line);
	my($rfile);
	my($file_key);
	my($cdate_time);
	my($cdate_key);
    while (defined ($line = <BULKLISTFILE>)) {
		chomp($line);
		if ($trace > 3) {
			print "bulklist line=$line\n";
		};
		if ($line =~ /file=/) {
			$rfile = &get_ins_file($line);
			$file_key = "file:$rfile";
			if ($trace > 3) {
				print "file_key=$file_key\n";
			};
		} elsif ($line =~ /#INS PROCESSED=/) {
			$line =~ s/^.*at //;
			$cdate_time = $line;
			$cdate_time =~ s#/#-#g;
			my($month, $day, $year) = split("-", &get_date($cdate_time));
			$cdate_key = "date:" . ($year + 2000) . "-$month-$day";
			if ($trace > 3) {
				print "cdate_key=$cdate_key\n";
			};
		} elsif ($line =~ /End File/) {
			if ($trace > 3) {
				print "end file=$line\n";
			};
			if (!defined ($line = <BULKLISTFILE>)) {
				last;
			};
			chomp($line);
			if ($line =~ /^\d\d*$/) {
				if ($trace > 3) {
					print "count=$line\n";
				};
				$act_bif_name_date_created{$file_key} = $cdate_key;
				$act_bif_name_counts{$file_key} = $line;
			};
		};
	}
    close (BULKLISTFILE);
}

sub proc_log_line {
	my ($line, $ptr_continue) = @_;
	if ($trace > 3) {
		print "line=$line\n";
	};
	if ($line =~ /Opening ins .*=.*upd_.*ins/) {
		&proc_opening_ins($line);
	} elsif ($line =~ /Finishing lang=.*files/) {
		&proc_finishing_ins($line);
	} elsif ($line =~ /^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3} FATAL /) {
		&proc_fatal_ins($line);
	} elsif ($line =~ /Indexer Daemon . file=.*-upd_/) {
		&proc_starting_pgm_ins($line);
	} elsif ($line =~ / IncrIndexer\d+ - coll bif=.*upd_.*ins/) {
		&proc_indexing_ins($line);
		if ($short_test && ($short_test++ > $max_short_test)) {
			return;
		}
	} elsif ($line =~ / IncrIndexer.*removed.*=.*upd_.*ins/) {
		&proc_removing_ins_indexer($line);
		if ($short_test && ($short_test++ > $max_short_test)) {
			return;
		}
	} elsif ($line =~ / IncrRenamer.*delete.*=.*upd_.*ins/) {
		&proc_removing_ins_remover($line);
		if ($short_test && ($short_test++ > $max_short_test)) {
			return;
		}
	} elsif ($line =~ / IncrGatherFiles.*coll index=/) {
		&proc_coll_index($line);
	} elsif ($line =~ / IncrGatherFiles.*update key/) {
		&proc_upd_key($line);
	} elsif ($shutdown &
			($line =~ /^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3} INFO /)) {
		$shutdown = 0;
		&proc_info_line($line);
	}
}

sub get_ins_file {
	my ($line) = @_;
	my($file);
	if ((my($epos) = index ($line, "=")) > 0) {
		$file = substr ($line, $epos + 1);
		chomp($file);
		if ($trace > 3) {
			print "file=$file\n";
		};
	} else {
		return "";
	}
	my($rfile);
	if ((my($wpos) = rindex ($file, $WORKINPROGRESS)) > 0) {
		$rfile = substr ($file, $wpos + $WORKINPROGRESS_LEN + 1);
		if (!defined($rfile) || ($rfile eq "")) {
			&print_error("Error getting ins $line\n");
			return;
		}
		if ($trace > 3) {
			print "rfile=$rfile\n";
		};
		return $rfile;
	} else {
		return "";
	}
}

sub get_coll_lang {
	my ($rfile, $ptr_collix, $ptr_vlang) = @_;
	if ((my($upos) = rindex ($rfile, "_")) > 0) {
		$$ptr_collix = substr ($rfile, $upos + 1, 1);
		$$ptr_vlang = substr ($rfile, $upos - 2, 2);
		if ($trace > 3) {
			print "collix=$$ptr_collix\n";
			print "vlang=$$ptr_vlang\n";
		};
	};
}

sub proc_opening_ins {
	my ($line) = @_;
	if ($trace) {
		print "opened ins line=$line\n";
	};
	my($rfile) = &get_ins_file($line);
	if (!defined($rfile) || ($rfile eq "")) {
		&print_error("Error no file $line\n");
		return;
	}
	&get_coll_lang($rfile, \$collix, \$vlang);
	if (!defined($collix) || ($collix eq "") ||
			!defined($vlang) || ($vlang eq "")) {
		&print_error("Error no collix and vlang $line\n");
		return;
	}
	if ($trace > 3) {
		print "f collix=$collix\n";
		print "f vlang=$vlang\n";
	};
	my($cdate_time) = &get_date_time($line);
	my($chour_key) = "hour:" . $cdate_time;
	my($cdate) = &get_date($cdate_time);
	my($cdate_key) = "date:" . $cdate;
	if (!exists($gather_bif_date_counts{$cdate_key})) {
		$gather_bif_date_counts{$cdate_key} = 0;
	}
	$gather_bif_date_counts{$cdate_key}+= 1;
	if ($hourly_report) {
		if (!exists($gather_bif_hour_counts{$cdate_key})) {
			$gather_bif_hour_counts{$chour_key} = 0;
		}
		$gather_bif_hour_counts{$chour_key}+= 1;
	}
	my($gather_key) = "vlang:" . $cdate . ":" . $vlang;
	if (!exists($gather_bif_date_counts{$gather_key})) {
		$gather_bif_date_counts{$gather_key} = 0;
	}
	$gather_bif_date_counts{$gather_key}+= 1;
	my($gather_hour_key) = "vlang:" . $cdate_time . ":" . $vlang;
	if ($hourly_report) {
		if (!exists($gather_bif_date_counts{$gather_hour_key})) {
			$gather_bif_hour_counts{$gather_hour_key} = 0;
		}
		$gather_bif_hour_counts{$gather_hour_key}+= 1;
	}
	my($thread);
	if (!defined($thread = &get_thread($line))) {
		&print_error("Error opened bif no thread $line\n");
		return;
	}
	my($coll_key) = "coll:$cdate:$vlang:$collix";
	if (!exists($gather_bif_date_counts{$coll_key})) {
		$gather_bif_date_counts{$coll_key} = 0;
	}
	$gather_bif_date_counts{$coll_key}+= 1;
	my($coll_hour_key) = "coll:$cdate_time:$vlang:$collix";
	if ($hourly_report) {
		if (!exists($gather_bif_hour_counts{$coll_hour_key})) {
			$gather_bif_hour_counts{$coll_hour_key} = 0;
		}
		$gather_bif_hour_counts{$coll_hour_key}+= 1;
	}
	my($open_key) = "coll:" . $vlang . ":" . $collix;
	if (exists($open_bif_names{$open_key}) &&
		!exists($bif_name_date_closed{$open_bif_names{$open_key}})) {
		$bif_name_date_closed{$open_bif_names{$open_key}} = $cdate;
	}
	if ($hourly_report) {
		if (exists($open_bif_names{$open_key}) &&
			!exists($bif_name_hour_closed{$open_bif_names{$open_key}})) {
			$bif_name_hour_closed{$open_bif_names{$open_key}} = $cdate_time;
		}
	}
	my($file_key) = "file:$rfile";
	$open_bif_names{$open_key} = $file_key;
	$bif_name_colls{$file_key} = $open_key;
	if ($trace) {
		print "Open bif open_key=$open_key\n";
	};
	
	$bif_name_date_created{$file_key} = $cdate;
	if ($hourly_report) {
		$bif_name_hour_created{$file_key} = $cdate_time;
	}
	$bif_name_proc{$file_key} = 0;
	$bif_name_counts{$file_key} = 0;
}

sub proc_finishing_ins {
	my ($line) = @_;
	if ($trace) {
		print "closed finishing line=$line\n";
	};
	my($vlang);
	if ((my($epos) = index ($line, "=")) > 0) {
		$vlang = substr ($line, $epos + 1, 2);
		if ($trace > 3) {
			print "c vlang=$vlang\n";
		}
	} else {
		&print_error("Error closed no eq $line\n");
		return;
	}
	my($cdate_time) = &get_date_time($line);
	my($cdate) = &get_date($cdate_time);
	for (my($ic) = 0; $ic < $vlang_counts{$vlang}; $ic++) {
		my($coll_key) = "coll:$vlang:$ic";
		if (exists($open_bif_names{$coll_key})) {
			$bif_name_date_closed{$open_bif_names{$coll_key}} = $cdate;
			if ($hourly_report) {
				$bif_name_hour_closed{$open_bif_names{$coll_key}} = $cdate_time;
			}
			if ($trace > 3) {
				print "closed coll=$coll_key\n";
				print "closed file=$open_bif_names{$coll_key}\n";
			}
		}
	}

}

sub proc_info_line {
	my ($line) = @_;
	if ($trace) {
		print "info line=$line\n";
	};
	my($dpos) = index ($line, " - ");
	my($csec_key) = "sec:" . &get_date_time_secs($line);
	if ($dpos < 0) {
		&print_warning("No dash in INFO line.");
		$indexer_fatal_shutdown{$csec_key} = "info:Unknown";
	} else {
		$indexer_fatal_shutdown{$csec_key} = "info:" .
			    substr($line, $dpos + 3);
	}
}

sub proc_fatal_ins {
	my ($line) = @_;
	if ($trace) {
		print "closed fatal line=$line\n";
	};
	my($cdate_time) = &get_date_time($line);
	my($chour_key) = "hour:" . $cdate_time;
	my($cdate) = &get_date($cdate_time);
	$shutdown = 1;
	my($dpos) = index ($line, " - ");
	my($csec_key) = "sec:" . &get_date_time_secs($line);
	if ($dpos < 0) {
		&print_warning("No dash in FATAL line.");
		$indexer_fatal_shutdown{$csec_key} = "exception:Unknown";
	} else {
		$indexer_fatal_shutdown{$csec_key} = "exception:" .
			    substr($line, $dpos + 3);
	}
	foreach $coll_key (keys(%open_bif_names)) {
		if (!exists($bif_name_date_closed{$open_bif_names{$coll_key}})) {
			$bif_name_date_closed{$open_bif_names{$coll_key}} = $cdate;
		}
		if ($hourly_report) {
			if (!exists($bif_name_hour_closed{$open_bif_names{$coll_key}})) {
				$bif_name_hour_closed{$open_bif_names{$coll_key}} = $cdate_time;
			}
		}
	}
}

sub proc_starting_pgm_ins {
	my ($line) = @_;
	if ($trace) {
		print "starting ins line=$line\n";
	};
	my($rfile) = &get_ins_file($line);
	if (!defined($rfile) || ($rfile eq "")) {
		&print_error("Error starting no file $line\n");
		return;
	}
	my($file_key) = "file:$rfile";
	my($cdate_time) = &get_date_time($line);
	my($chour_key) = "hour:" . $cdate_time;
	my($cdate) = &get_date($cdate_time);
	if (!exists($bif_name_proc{$file_key})) {
		if ($trace > 1) {
			print "Warning starting bif from previous file not created:  $rfile\n";
		};
		return;
	}
	if (!exists($bif_name_date_closed{$file_key})) {
		$bif_name_date_closed{$file_key} = $cdate;
	}
	if ($hourly_report) {
		if (!exists($bif_name_hour_closed{$file_key})) {
			$bif_name_hour_closed{$file_key} = $cdate_time;
		}
	}
	if (exists($bif_name_date_started_indexing{$file_key})) {
		delete $bif_name_date_started_indexing{$file_key};
	}
	if ($hourly_report) {
		if (exists($bif_name_hour_started_indexing{$file_key})) {
			delete $bif_name_hour_started_indexing{$file_key};
		}
	}
}

sub proc_indexing_ins {
	my ($line) = @_;
	if ($trace) {
		print "indexing ins line=$line\n";
	};
	my($rfile) = &get_ins_file($line);
	my($file_key) = "file:$rfile";
	&get_coll_lang($rfile, \$collix, \$vlang);
	if (!defined($collix) || ($collix eq "") ||
			!defined($vlang) || ($vlang eq "")) {
		&print_error("Error no collix and vlang $line\n");
		return;
	}
	if ($trace > 3) {
		print "i collix=$collix\n";
		print "i vlang=$vlang\n";
	};
	my($thread);
	if (!defined($thread = &get_thread($line))) {
		&print_error("Error indexed bif no thread $line\n");
		return;
	}
	#TODO thread changed
	my($coll_thread_key) = "coll:" . $vlang . ":" . $collix;
	if (exists($coll_threads{$coll_thread_key})) {
		if ($coll_threads{$coll_thread_key} ne "$thread") {
			$prev_coll_threads{$coll_thread_key} =
				$coll_threads{$coll_thread_key};
			$coll_threads{$coll_thread_key} = $thread;
		}
	} else {
		$coll_threads{$coll_thread_key} = $thread;
	}
	my($cdate_time) = &get_date_time($line);
	my($chour_key) = "hour:" . $cdate_time;
	my($cdate) = &get_date($cdate_time);
	my($cdate_key) = "date:" . $cdate;
	if (exists($bif_name_date_started_indexing{$file_key})) {
		print "Error indexing already started.\n";
		&print_error("Error line $line.");
		return;
	}
	$bif_name_date_started_indexing{$file_key} = $cdate;
	if ($hourly_report) {
		$bif_name_hour_started_indexing{$file_key} = $cdate_time;
	}
}

sub proc_removing_ins_indexer {
	my ($line) = @_;
	if ($trace) {
		print "removed ins indexer line=$line\n";
	};
	if ((my($ppos) = rindex ($line, "|")) > 0) {
		$line = substr ($line, 0, $ppos);
		if ($trace > 3) {
			print "line=$line\n";
		}
	}
	my($rfile) = &get_ins_file($line);
	my($file_key) = "file:$rfile";
	&get_coll_lang($rfile, \$collix, \$vlang);
	if (!defined($collix) || ($collix eq "") ||
			!defined($vlang) || ($vlang eq "")) {
		&print_error("Error no collix and vlang $line\n");
		return;
	}
	if ($trace > 3) {
		print "ir collix=$collix\n";
		print "ir vlang=$vlang\n";
	};
	my($thread);
	if (!defined($thread = &get_thread($line))) {
		&print_error("Error indexed bif no thread $line\n");
		return;
	}
	my($coll_thread_key) = "coll:" . $vlang . ":" . $collix;
	if (exists($coll_threads{$coll_thread_key})) {
		if ($coll_threads{$coll_thread_key} ne "$thread") {
			$prev_coll_threads{$coll_thread_key} =
				$coll_threads{$coll_thread_key};
			$coll_threads{$coll_thread_key} = $thread;
		}
	} else {
		$coll_threads{$coll_thread_key} = $thread;
	}
	if (!exists($bif_name_proc{$file_key})) {
		if ($trace > 1) {
			print "Warning removed bif from previous file not processed:  $rfile\n";
		};
		return;
	}
	if (!exists($bif_name_counts{$file_key})) {
		&print_error("Error indexed bif has no count:  $rfile\n");
		return;
	}
	$bif_name_proc{$file_key} = 1;
	my($cdate_time) = &get_date_time($line);
	my($chour_key) = "hour:" . $cdate_time;
	my($cdate) = &get_date($cdate_time);
	my($cdate_key) = "date:" . $cdate;
	if (!exists($indexed_date_counts{$cdate_key})) {
		$indexed_date_counts{$cdate_key} = 0;
	}
	$indexed_date_counts{$cdate_key} += $bif_name_counts{$file_key};
	if ($hourly_report) {
		if (!exists($indexed_hour_counts{$chour_key})) {
			$indexed_hour_counts{$chour_key} = 0;
		}
		$indexed_hour_counts{$chour_key} += $bif_name_counts{$file_key};
	}
	if (!exists($indexed_bif_date_counts{$cdate_key})) {
		$indexed_bif_date_counts{$cdate_key} = 0;
	}
	$indexed_bif_date_counts{$cdate_key}++;
	if ($hourly_report) {
		if (!exists($indexed_bif_hour_counts{$chour_key})) {
			$indexed_bif_hour_counts{$chour_key} = 0;
		}
		$indexed_bif_hour_counts{$chour_key}++;
	}
	$bif_name_date_indexed{$file_key} = $cdate;
	if ($hourly_report) {
		$bif_name_hour_indexed{$file_key} = $cdate_time;
	}
	my($lang_key) = "vlang:" . $cdate . ":" . $vlang;
	if (!exists($indexed_date_counts{$lang_key})) {
		$indexed_date_counts{$lang_key} = 0;
	}
	$indexed_date_counts{$lang_key} += $bif_name_counts{$file_key};
	my($lang_hour_key) = "vlang:" . $cdate_time . ":" . $vlang;
	if ($hourly_report) {
		if (!exists($indexed_hour_counts{$lang_hour_key})) {
			$indexed_hour_counts{$lang_hour_key} = 0;
		}
		$indexed_hour_counts{$lang_hour_key} += $bif_name_counts{$file_key};
	}
	if (!exists($indexed_bif_date_counts{$lang_key})) {
		$indexed_bif_date_counts{$lang_key} = 0;
	}
	$indexed_bif_date_counts{$lang_key}++;
	if ($hourly_report) {
		if (!exists($indexed_bif_hour_counts{$lang_key})) {
			$indexed_bif_hour_counts{$lang_hour_key} = 0;
		}
		$indexed_bif_hour_counts{$lang_hour_key}++;
	}
	my($coll_key) = "coll:" . $cdate . ":" . $vlang . ":" . $collix;
	if (!exists($indexed_date_counts{$coll_key})) {
		$indexed_date_counts{$coll_key} = 0;
	}
	$indexed_date_counts{$coll_key} += $bif_name_counts{$file_key};
	my($coll_hour_key) = "coll:" . $cdate_time . ":" . $vlang . ":" . $collix;
	if ($hourly_report) {
		if (!exists($indexed_hour_counts{$coll_hour_key})) {
			$indexed_hour_counts{$coll_hour_key} = 0;
		}
		$indexed_hour_counts{$coll_hour_key} += $bif_name_counts{$file_key};
	}
	if (!exists($indexed_bif_date_counts{$coll_key})) {
		$indexed_bif_date_counts{$coll_key} = 0;
	}
	$indexed_bif_date_counts{$coll_key}++;
	if ($hourly_report) {
		if (!exists($indexed_bif_hour_counts{$coll_hour_key})) {
			$indexed_bif_hour_counts{$coll_hour_key} = 0;
		}
		$indexed_bif_hour_counts{$coll_hour_key}++;
	}
	my($thread_key) = "thread:" . $cdate . ":" . $thread;
	if (!exists($indexed_date_counts{$thread_key})) {
		$indexed_date_counts{$thread_key} = 0;
	}
	$indexed_date_counts{$thread_key} += $bif_name_counts{$file_key};
	my($thread_hour_key) = "thread:" . $cdate_time . ":" . $thread;
	if ($hourly_report) {
		if (!exists($indexed_hour_counts{$thread_hour_key})) {
			$indexed_hour_counts{$thread_hour_key} = 0;
		}
		$indexed_hour_counts{$thread_hour_key} += $bif_name_counts{$file_key};
	}
	if (!exists($indexed_bif_date_counts{$thread_key})) {
		$indexed_bif_date_counts{$thread_key} = 0;
	}
	$indexed_bif_date_counts{$thread_key}++;
	if ($hourly_report) {
		if (!exists($indexed_bif_hour_counts{$thread_hour_key})) {
			$indexed_bif_hour_counts{$thread_hour_key} = 0;
		}
		$indexed_bif_hour_counts{$thread_hour_key}++;
	}
}

sub proc_removing_ins_remover {
	my ($line) = @_;
	if ($trace) {
		print "removed ins remover line=$line\n";
	};
	my($rfile) = &get_ins_file($line);
	my($file_key) = "file:$rfile";
	&get_coll_lang($rfile, \$collix, \$vlang);
	if (!defined($collix) || ($collix eq "") ||
			!defined($vlang) || ($vlang eq "")) {
		&print_error("Error no collix and vlang $line\n");
		return;
	}
	if ($trace > 3) {
		print "rr collix=$collix\n";
		print "rr vlang=$vlang\n";
	};
	if (!exists($bif_name_proc{$file_key})) {
		if ($trace > 1) {
			print "Warning indexed bif from previous file not processed:  $rfile\n";
		};
		return;
	} elsif ($bif_name_proc{$file_key}) {
		if ($trace > 1) {
			print "OK removing indexed bif:  $rfile\n";
		};
		return;
	}
	if (!exists($bif_name_counts{$file_key})) {
		if ($trace > 1) {
			&print_error("Error indexed bif has no count:  $rfile\n");
		};
		return;
	}
	my($coll_thread_key) = "coll:" . $vlang . ":" . $collix;
	my($thread) = $coll_threads{$coll_thread_key};
	if (!defined($thread) || ($thread eq "")) {
		&print_error("Error removed bif no thread $line\n");
		return;
	}
	my($cdate_time) = &get_date_time($line);
	my($chour_key) = "hour:" . $cdate_time;
	my($cdate) = &get_date($cdate_time);
	my($cdate_key) = "date:" . $cdate;
	if (!exists($lost_date_counts{$cdate_key})) {
		$lost_date_counts{$cdate_key} = 0;
	}
	$lost_date_counts{$cdate_key} += $bif_name_counts{$file_key};
	if ($hourly_report) {
		if (!exists($lost_date_counts{$chour_key})) {
			$lost_date_counts{$chour_key} = 0;
		}
		$lost_date_counts{$chour_key} += $bif_name_counts{$file_key};
	}
	if (!exists($lost_date_counts{$cdate_key})) {
		$lost_date_counts{$cdate_key} = 0;
	}
	$bif_name_date_indexed{$file_key} = $cdate;
	if ($hourly_report) {
		$bif_name_hour_indexed{$file_key} = $cdate_time;
	}
	my($lang_key) = "vlang:" . $cdate . ":" . $vlang;
	if (!exists($lost_date_counts{$lang_key})) {
		$lost_date_counts{$lang_key} = 0;
	}
	$lost_date_counts{$lang_key} += $bif_name_counts{$file_key};
	my($lang_hour_key) = "vlang:" . $cdate_time . ":" . $vlang;
	if ($hourly_report) {
		if (!exists($lost_date_counts{$lang_hour_key})) {
			$lost_date_counts{$lang_hour_key} = 0;
		}
		$lost_date_counts{$lang_hour_key} += $bif_name_counts{$file_key};
	}
	if (!exists($lost_bif_date_counts{$lang_key})) {
		$lost_bif_date_counts{$lang_key} = 0;
	}
	$lost_bif_date_counts{$lang_key}++;
	if ($hourly_report) {
		if (!exists($lost_bif_date_counts{$lang_key})) {
			$lost_bif_date_counts{$lang_hour_key} = 0;
		}
		$lost_bif_date_counts{$lang_hour_key}++;
	}
	my($coll_key) = "coll:" . $cdate . ":" . $vlang . ":" . $collix;
	if (!exists($lost_date_counts{$coll_key})) {
		$lost_date_counts{$coll_key} = 0;
	}
	$lost_date_counts{$coll_key} += $bif_name_counts{$file_key};
	my($coll_hour_key) = "coll:" . $cdate_time . ":" . $vlang . ":" . $collix;
	if ($hourly_report) {
		if (!exists($lost_date_counts{$coll_hour_key})) {
			$lost_date_counts{$coll_hour_key} = 0;
		}
		$lost_date_counts{$coll_hour_key} += $bif_name_counts{$file_key};
	}
	if (!exists($lost_bif_date_counts{$coll_key})) {
		$lost_bif_date_counts{$coll_key} = 0;
	}
	$lost_bif_date_counts{$coll_key}++;
	if ($hourly_report) {
		if (!exists($lost_bif_date_counts{$coll_hour_key})) {
			$lost_bif_date_counts{$coll_hour_key} = 0;
		}
		$lost_bif_date_counts{$coll_hour_key}++;
	}
	my($thread_key) = "thread:" . $cdate . ":" . $thread;
	if (!exists($lost_date_counts{$thread_key})) {
		$lost_date_counts{$thread_key} = 0;
	}
	$lost_date_counts{$thread_key} += $bif_name_counts{$file_key};
	my($thread_hour_key) = "thread:" . $cdate_time . ":" . $thread;
	if ($hourly_report) {
		if (!exists($lost_date_counts{$thread_hour_key})) {
			$lost_date_counts{$thread_hour_key} = 0;
		}
		$lost_date_counts{$thread_hour_key} += $bif_name_counts{$file_key};
	}
	if (!exists($lost_bif_date_counts{$thread_key})) {
		$lost_bif_date_counts{$thread_key} = 0;
	}
	$lost_bif_date_counts{$thread_key}++;
	if ($hourly_report) {
		if (!exists($lost_bif_date_counts{$thread_hour_key})) {
			$lost_date_counts{$thread_hour_key} = 0;
		}
		$lost_bif_date_counts{$thread_hour_key}++;
	}
}

sub proc_coll_index {
	my ($line) = @_;
	if ($trace > 3) {
		print "coll line=$line\n";
	};
	my($collix);
	if ((my($epos) = index ($line, "=")) > 0) {
		$collix = substr ($line, $epos + 1);
		chomp($collix);
		if ($trace > 3) {
			print "g collix=$collix\n";
		};
	} else {
		return;
	}
	my($thread);
	if (!defined($thread = &get_thread($line))) {
		&print_error("Error coll index no thread $line\n");
		return;
	}
	my($thread_key) = "thread:" . $thread;
	$curr_doc_ix{$thread_key} = $collix;
}

sub proc_upd_key {
	my ($line) = @_;
	if ($trace > 2) {
		print "upd key line=$line\n";
	};
	my($doc_key);
	if ((my($epos) = index ($line, "=")) > 0) {
		$doc_key = substr ($line, $epos + 1);
		if ($trace > 3) {
			print "doc_key=$doc_key\n";
		};
	} else {
		&print_error("Error upd key no equals $line\n");
		return;
	}
	my($thread);
	my($vlang);
	if ((my($cpos) = rindex ($doc_key, "-")) > 0) {
		if (!defined($thread = &get_thread($line))) {
			&print_error("Error upd key no thread $line\n");
			return;
		}
		$vlang = substr ($doc_key, $cpos + 1, 2);
		$doc_key = substr ($doc_key, 0, $cpos);
		$thread_key = "thread:" . $thread;
		if (!exists($curr_doc_ix{$thread_key})) {
			if ($trace > 1) {
				print "g doc key from previos file not coll index:  " .
					   "$doc_key\n";
			}
			return;
		}
		my($open_key) = "coll:$vlang:$curr_doc_ix{$thread_key}"; #new
		if ($trace > 3) {
			print "g vlang=$vlang\n";
			print "doc_key 2=$doc_key\n";
			print "open_key=$open_key\n";
		};
		if (!exists($open_bif_names{$open_key})) {
			if ($trace > 1) {
				print "Doc id from previous file not opened: $open_key $line\n";
			};
			return;
		}
		if (!exists($bif_name_counts{$open_bif_names{$open_key}})) {
			&print_error("Error doc id must have count for open bif:  $open_bif_names{$open_key} $line\n");
			return;
		}
		$bif_name_counts{$open_bif_names{$open_key}}++;
		my($cdate) = &get_date($line);
		my($cdate_key) = "date:" . $cdate;
		if (!exists($gather_counts{$cdate_key})) {
			$gather_counts{$cdate_key} = 0;
		}
		$gather_counts{$cdate_key}+= 1;
		my($gather_key) = "vlang:" . $cdate . ":" . $vlang;
		if (!exists($gather_counts{$gather_key})) {
			$gather_counts{$gather_key} = 0;
		}
		$gather_counts{$gather_key}+= 1;
		my($coll_key) = "coll:$cdate:$vlang:$curr_doc_ix{$thread_key}";
		if (!exists($gather_counts{$coll_key})) {
			$gather_counts{$coll_key} = 0;
		}
		$gather_counts{$coll_key}+= 1;
	} else {
		&print_error("Error upd key no equals - $line\n");
		return;
	}
}

sub get_thread {
	my ($line) = @_;
	if (($dpos = index ($line, " DEBUG ")) <= 0) {
		return "";
	}
	$thread = substr ($line, $dpos + length(" DEBUG "));
	if (($spos = index ($thread, " ")) <= 0) {
		return "";
	}
	$thread = substr ($thread, $spos - 1, 1);
	if ($trace > 3) {
		print "thread=$thread\n";
	};
	return $thread;
}

sub get_date_time_secs {
	my ($line) = @_;
	if ((my($spos) = index ($line, " ")) > 0) {
		if ((my($s2pos) = index ($line, " ", $spos + 1)) > 0) {
			my($cdate_time_mins) = substr ($line, 0, $s2pos);
			$cdate_time_mins =~ s/.\d{3}$//;
			if ($trace > 3) {
				print "cdate_time_mins=$cdate_time_mins\n";
			};
			return $cdate_time_mins;
		}
	}
	return "";
}

sub get_date_time {
	my ($line) = @_;
	if ((my($spos) = index ($line, " ")) > 0) {
		if ((my($s2pos) = index ($line, " ", $spos + 1)) > 0) {
			my($cdate_time) = substr ($line, 0, $s2pos);
			$cdate_time =~ s/:\d\d:\d\d.\d{3}$//;
			if ($trace > 3) {
				print "cdate_time=$cdate_time\n";
			};
			return $cdate_time;
		}
	}
	return "";
}

sub get_hour {
	my ($cdate_time) = @_;
	if (!defined($cdate_time) || ($cdate_time eq "")) {
		&print_error("Error date_time=$cdate_time\n");
		return "";
	}
	if ((my($spos) = index ($cdate_time, " ")) > 0) {
		my($chour) = substr ($cdate_time, $spos + 1);
		if ($trace > 3) {
			print "chour=$chour\n";
		};
		return $chour;
	} else {
		return "";
	}
}

sub get_all_hours {
	my ($cdate_time1, $cdate_time2) = @_;
	my(@rtn_hours) = ();
	my($curr_date_time) = $cdate_time1;
	my($curr_date) = &get_date($cdate_time1);
	my($curr_hour) = &get_hour($cdate_time1);
	while ($curr_date_time ne "$cdate_time2") {
		push(@rtn_hours, $curr_date_time);
		$curr_hour++;
		if ($curr_hour == 24) {
			my($cyear, $cmonth, $cday) = split("-", $curr_date);
			my($ltime) = timelocal(0, 0, 0, $cday, ($cmonth - 1), $cyear, 0, 0);
			$ltime += 24*60*60;
			my($nsecs, $nmin, $nhrs, $nmdays, $nmonth, $nyear, $nwdays, $nydays,
			   $ndaylight) = localtime($ltime);
			if ($nyear > 100) {
				$nyear -= 100;
				$nyear += 2000;
			}
			$curr_date = sprintf("%02d-%02d-%02d", $nyear, ($nmonth + 1),
								 $nmdays);
			$curr_hour = 0;
		}
		$curr_date_time = "$curr_date " . sprintf("%02d", $curr_hour);
		if ($trace > 3) {
			print "curr_date_time=$curr_date_time\n";
		};
	}
	push(@rtn_hours, $cdate_time2);
	return @rtn_hours;
}

sub get_date {
	my ($cdate_time) = @_;
	if (!defined($cdate_time) || ($cdate_time eq "")) {
		&print_error("Error date_time=$cdate_time\n");
		return "";
	}
	if ((my($spos) = index ($cdate_time, " ")) > 0) {
		my($cdate) = substr ($cdate_time, 0, $spos);
		if ($trace > 3) {
			print "cdate=$cdate\n";
		};
		return $cdate;
	} else {
		return "";
	}
}

sub print_error {
	my($err_msg) = @_;
	print "Error.  Line #$line_no.  File $log_file is log file.\n";
	print "$err_msg";
}

sub print_warning {
	my($err_msg) = @_;
	print "Warning.  Line #$line_no.  File $log_file is log file.\n";
	print "$err_msg";
}

sub print_hash {
	my($file, $print_label, $hash_ref) = @_;
	print $file "$print_label\n";
	print $file "Variable: ${$assoc_symbols{$hash_ref}}\n";
	print $file "Key: $key_descriptions{$hash_ref}\n";
	print $file "Value: $value_descriptions{$hash_ref}\n";
	print $file "$key_names{$hash_ref} $value_names{$hash_ref}\n";
	my($total) = 0;
	my($last_comp_key) = "";
	my(%minor_totals) = ();
	foreach $key (sort(keys(%$hash_ref))) {
		print $file "$key $hash_ref->{$key}\n";
		if ($value_names{$hash_ref} eq "count") {
			$total += $hash_ref->{$key};
			if ($key =~ /:[^:][^:]*:/) {
				my($total_key) = $key;
				$total_key =~ s/:[^:][^:]*:/:/;
				if (!exists($minor_totals{$total_key})) {
					$minor_totals{$total_key} = 0;
				}
				$minor_totals{$total_key} += $hash_ref->{$key};
			}
		}
	}
	$num_minor_keys = keys(%minor_totals);

	my(%sub_minor_totals) = ();
	foreach $total_key (sort(keys(%minor_totals))) {
		print $file "$total_key $minor_totals{$total_key}\n";
		if ($total_key =~ /:/) {
			my($sub_total_key) = $total_key;
			$sub_total_key =~ s/:.*$//;
			if (!exists($sub_minor_totals{$sub_total_key})) {
				$sub_minor_totals{$sub_total_key} = 0;
			}
			$sub_minor_totals{$sub_total_key} += $minor_totals{$total_key};
		}
	}
	foreach $total_key (sort(keys(%sub_minor_totals))) {
		print $file "$total_key $sub_minor_totals{$total_key}\n";
	}
	if ($num_minor_keys == 0) {
		if ($value_names{$hash_ref} eq "count") {
			print $file "Grand total: $total\n";
		}
	}
}

sub print_results {

&print_hash(\*STDOUT, "Doc collindexes:", \%curr_doc_ix);

&print_hash(\*DIAGFILE, "Opened files:", \%open_bif_names);

&print_hash(\*DIAGFILE, "BIF files created on date:", \%bif_name_date_created);

if (defined($bulkins_list)) {
	&print_hash(\*DIAGFILE, "Actual bif created:", \%act_bif_name_date_created);
}

if ($hourly_report) {
	&print_hash(\*DIAGFILE, "BIF files created on hour:", \%bif_name_hour_created);
}

&print_hash(\*DIAGFILE, "BIF files closed date:", \%bif_name_date_closed);

if ($hourly_report) {
	&print_hash(\*DIAGFILE, "BIF files closed hour:", \%bif_name_hour_closed);
}

&print_hash(\*STDOUT, "Exception/info date:", \%indexer_fatal_shutdown);

&print_hash(\*DIAGFILE, "BIF files indexed date:", \%bif_name_date_indexed);

if ($hourly_report) {
	&print_hash(\*DIAGFILE, "BIF files indexed hour:", \%bif_name_hour_indexed);
}

&print_hash(\*DIAGFILE, "BIF files colls:", \%bif_name_colls);

if (defined($bulkins_list)) {
	&print_hash(\*DIAGFILE, "Actual BIF counts:", \%act_bif_name_counts);
}

&print_hash(\*DIAGFILE, "Biff file counts:", \%bif_name_counts);

&print_hash(\*DIAGFILE, "Biff file processed:", \%bif_name_proc);

&print_hash(\*STDOUT, "Gather bif counts date:", \%gather_bif_date_counts);

if ($hourly_report) {
	&print_hash(\*STDOUT, "Gather bif counts hour:", \%gather_bif_hour_counts);
}

&print_hash(\*STDOUT, "Gather counts:", \%gather_counts);

%indexed_incr_hour_counts = ();
$assoc_symbols{\%indexed_incr_hour_counts} = \indexed_incr_hour_counts;
$key_names{\%indexed_incr_hour_counts} = "hour";
$key_descriptions{\%indexed_incr_hour_counts} = "Hour of indexing increment.";
$value_names{\%indexed_incr_hour_counts} = "count";
$value_descriptions{\%indexed_incr_hour_counts} =
	"Indexing incremental doc counts per hour.";
if ($hourly_report) {
	print "Calculating indexing incremental hour counts...:\n";
	foreach $bif_name (sort(keys(%bif_name_proc))) {
		if ($bif_name_proc{$bif_name}) {
			if (!exists($bif_name_hour_started_indexing{$bif_name}) ||
				($bif_name_hour_started_indexing{$bif_name} eq "")) {
				&print_error("Error file no start $bif_name\n");
				next;
			}
			if (!exists($bif_name_hour_indexed{$bif_name}) ||
				($bif_name_hour_indexed{$bif_name} eq "")) {
				&print_error("Error file no end $bif_name\n");
				next;
			}
			@hours = &get_all_hours($bif_name_hour_started_indexing{$bif_name},
									$bif_name_hour_indexed{$bif_name});
			my($incr_docs) = floor($bif_name_counts{$bif_name} / @hours);
			my($extra) =  $bif_name_counts{$bif_name} - ($incr_docs * @hours);
			foreach $hour (@hours) {
				if (!exists($indexed_incr_hour_counts{$hour})) {
					$indexed_incr_hour_counts{$hour} = 0;
				}
				$indexed_incr_hour_counts{$hour} += $incr_docs;
			}
			$indexed_incr_hour_counts{$bif_name_hour_indexed{$bif_name}} += $extra;
		}
	}

	&print_hash(\*STDOUT, "Incremental indexed docs per hour:",
				\%indexed_incr_hour_counts);
}
%backlog_colls_counts = ();
$assoc_symbols{\%backlog_colls_counts} = \backlog_colls_counts;
$key_names{\%backlog_colls_counts} = "coll";
$key_descriptions{\%backlog_colls_counts} = "Collection of backlog.";
$value_names{\%backlog_colls_counts} = "count";
$value_descriptions{\%backlog_colls_counts} =
	"Backlog doc counts per collection.";
print "Calculating bif file backlog...:\n";
foreach $bif_name (sort(keys(%bif_name_proc))) {
	if (!$bif_name_proc{$bif_name}) {
		$coll = $bif_name_colls{$bif_name};
		if (!exists($backlog_colls_counts{$coll})) {
			$backlog_colls_counts{$coll} = 0;
		}
		$backlog_colls_counts{$coll} += $bif_name_counts{$bif_name};
	}
}

&print_hash(\*STDOUT, "Backlog per coll:", \%backlog_colls_counts);

&print_hash(\*STDOUT, "Coll running threads:", \%coll_threads);

%backlog_threads_counts = ();
$assoc_symbols{\%backlog_threads_counts} = \backlog_threads_counts;
$key_names{\%backlog_threads_counts} = "thread";
$key_descriptions{\%backlog_threads_counts} = "Thread of the backlog.";
$value_names{\%backlog_threads_counts} = "count";
$value_descriptions{\%backlog_threads_counts} =
		"Backlog doc counts per thread.";
print "Calculating biff file per thread backlog...:\n";
foreach $coll_key (sort(keys(%backlog_colls_counts))) {
	if (!exists($coll_threads{$coll_key})) {
		print "Warning.  Thread for coll $coll_key not known, skipping.\n";
		next;
	}

	$thread_key = $coll_threads{$coll_key};
	if (!exists($backlog_threads_counts{$thread_key})) {
		$backlog_threads_counts{$thread_key} = 0;
	}
	$backlog_threads_counts{$thread_key} += $backlog_colls_counts{$coll_key};
}

&print_hash(\*STDOUT, "Backlog per thread counts:", \%backlog_threads_counts);

%backlog_bif_dates = ();
$assoc_symbols{\%backlog_bif_dates} = \backlog_bif_dates;
$key_names{\%backlog_bif_dates} = "date";
$key_descriptions{\%backlog_bif_dates} = "Date(incremental) of backlog bifs.";
$value_names{\%backlog_bif_dates} = "count";
$value_descriptions{\%backlog_bif_dates} = "Backlog(incremental) bifs per date.";

%backlog_dates = ();
$assoc_symbols{\%backlog_dates} = \backlog_dates;
$key_names{\%backlog_dates} = "date";
$key_descriptions{\%backlog_dates} = "Date(incremental) of backlog docs.";
$value_names{\%backlog_dates} = "count";
$value_descriptions{\%backlog_dates} = "Backlog(incremental) docs per date.";

print "Calculating backlog incremental(closed)...:\n";
foreach $file_key (sort(keys(%bif_name_date_closed))) {
	$date_key = $bif_name_date_closed{$file_key};
	if (!exists($backlog_dates{$date_key})) {
		$backlog_dates{$date_key} = 0;
	}
	if (!exists($backlog_bif_dates{$date_key})) {
		$backlog_bif_dates{$date_key} = 0;
	}
	$backlog_dates{$date_key} += $bif_name_counts{$file_key};
	$backlog_bif_dates{$date_key}++;
}

print "Calculating backlog incremental(indexed)...:\n";
foreach $file_key (sort(keys(%bif_name_date_indexed))) {

	$date_key = $bif_name_date_indexed{$file_key};
	if (!exists($backlog_dates{$date_key})) {
		next;
	}
	if (!exists($backlog_bif_dates{$date_key})) {
		next;
	}
	$backlog_dates{$date_key} -= $bif_name_counts{$file_key};
	$backlog_bif_dates{$date_key}--;
}

&print_hash(\*STDOUT, "Incremental backlog bifs per date counts (NOT cumulative):",
			\%backlog_bif_dates);

&print_hash(\*STDOUT, "Incremental backlog docs per date counts (NOT cumulative):",
			\%backlog_dates);

&print_hash(\*STDOUT, "Indexing bif date counts:", \%indexed_bif_date_counts);

if ($hourly_report) {
	&print_hash(\*STDOUT, "Indexing bif hour counts:", \%indexed_bif_hour_counts);
}

&print_hash(\*STDOUT, "Indexing doc date counts:", \%indexed_date_counts);

if ($hourly_report) {
	&print_hash(\*STDOUT, "Indexing doc hour counts:", \%indexed_hour_counts);
}

&print_hash(\*STDOUT, "Lost bif date counts:", \%lost_bif_date_counts);

if ($hourly_report) {
	&print_hash(\*STDOUT, "Lost bif hour counts:", \%lost_bif_hour_counts);
}

&print_hash(\*STDOUT, "Lost doc date counts:", \%lost_date_counts);

if ($hourly_report) {
	&print_hash(\*STDOUT, "Lost doc hour counts:", \%lost_date_counts);
}

if ($hourly_report) {
	&print_hash(\*STDOUT, "Lost bif hour counts:", \%lost_bif_hour_counts);
}

}

print "$test1 $test2 $test3 $test4 $test5 $test6 $test7 $test8 $testcount $use_testdata\n";
